package com.xq.utils;

import com.xq.annotation.JsoupDocument;
import com.xq.annotation.Extract;
import com.xq.annotation.JsoupDocument;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.lang.reflect.Field;
import java.util.List;

public final class CrawlerUtils {

    private CrawlerUtils() {
    }

    /**
     * 获取connection
     *
     * @param url
     * @return
     */
    public static Connection getConnection(String url) {
        Connection connection = Jsoup.connect(url);
        // 3.伪造请求头
        connection.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        connection.header("Accept-Encoding", "gzip, deflate, br");
        connection.header("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        connection.header("Cache-Control", "max-age=0");
        connection.header("Connection", "keep-alive");
        connection.header("Cookie", "Hm_lvt_ec661610f14acf2457496da3a87d804d=1592569443; Hm_lpvt_ec661610f14acf2457496da3a87d804d=1592569443");
        connection.header("Host", "zhipeng0908.gitee.io");
        connection.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0");
        return connection;
    }

    /**
     * 执行爬虫程序
     *
     * @param clz 目标类
     * @param url 目标url
     * @param obj 获取结果，是pojo还是list
     * @throws Exception
     */
    public static Object execute(Class<?> clz, String url, Object obj) throws Exception {
        // 获取jsoupDocument对象
        JsoupDocument jsoupDocument = clz.getAnnotation(JsoupDocument.class);
        // 判断注解不为空
        if (jsoupDocument == null) {
            throw new RuntimeException("该实体类不是爬虫实体！");
        }
        String targetUrl = jsoupDocument.targetUrl();
        // 如果存在targetUrl，那么爬取的url以targetUrl为主
        if (StringUtils.isNotBlank(targetUrl)) {
            url = targetUrl;
        }
        // 判断url是否为空
        if (StringUtils.isBlank(url)) {
            throw new RuntimeException("未指定爬取目标url！");
        }
        // 获取connection
        Connection connection = getConnection(url);

        // 获取爬取方式
        Connection.Method method = jsoupDocument.method();

        // 获取document
        Document document = connection.method(method).execute().parse();
        Element bodyElement = document.body();
        // 获取类上的css选择器
        String cssQuery = jsoupDocument.cssQuery();
        // 获取目标列表
        Field[] declaredFields = clz.getDeclaredFields();

        // 判断返回对象的类型
        if (obj instanceof List) {
            Elements entityElements = bodyElement.select(cssQuery);
            return handleListValue(clz, obj, entityElements, declaredFields);
        } else {
            return handlePojoValue(obj, bodyElement, declaredFields, cssQuery);
        }
    }

    /**
     * 处理实体类
     *
     * @param obj
     * @param bodyElement
     * @param declaredFields
     * @param cssQuery
     * @return
     * @throws IllegalAccessException
     */
    private static Object handlePojoValue(Object obj, Element bodyElement, Field[] declaredFields, String cssQuery) throws Exception {
        // 实体类,只取出第一条数据
        if (StringUtils.isBlank(cssQuery)) {
            setFieldValue(declaredFields, bodyElement, obj);
        } else {
            Elements entityElements = bodyElement.select(cssQuery);
            int size = entityElements.size();
            if (size == 0) {
                return obj;
            }
            Element element = entityElements.get(0);
            setFieldValue(declaredFields, element, obj);
        }
        return obj;
    }

    /**
     * 处理集合类型
     *
     * @param clz
     * @param obj
     * @param entityElements
     * @param declaredFields
     * @return
     * @throws InstantiationException
     * @throws IllegalAccessException
     */
    private static List handleListValue(Class<?> clz, Object obj, Elements entityElements, Field[] declaredFields) throws Exception {
        List list = (List) obj;
        // 是集合
        for (Element element : entityElements) {
            // 创建对象
            Object object = clz.newInstance();
            setFieldValue(declaredFields, element, object);
            list.add(object);
        }
        return list;
    }

    /**
     * 设置字段值
     *
     * @param declaredFields 字段对象
     * @param element        元素
     * @param object         获取对象
     * @throws IllegalAccessException
     */
    private static void setFieldValue(Field[] declaredFields, Element element, Object object) throws Exception {
        for (Field field : declaredFields) {
            // 暴力反射
            field.setAccessible(true);
            // 获取JsoupDocument注解
            JsoupDocument fieldJsoupDocument = field.getAnnotation(JsoupDocument.class);
            if(fieldJsoupDocument != null) {
                // 递归爬取
                Class<?> fieldType = field.getType();
                Object fieldObject = fieldType.newInstance();
                // 获取url
                String hrefAttr = fieldJsoupDocument.hrefAttr();
                // 判断hrefAttr是否为空
                if(StringUtils.isBlank(hrefAttr)) {
                    throw new RuntimeException("字段没"+field.getName()+"有指定哪个字段中获取目标url");
                }
                // 获取指定指定
                Class<?> objectClass = object.getClass();
                Field hrefField = objectClass.getDeclaredField(hrefAttr);
                hrefField.setAccessible(true);
                // 获取该字段的值
                String hrefValue = (String) hrefField.get(object);

                // 处理域名
                if(!hrefValue.toLowerCase().startsWith("http")) {
                    // 不以http开头， 前面就拼接域名
                    String domain = fieldJsoupDocument.domain();
                    hrefValue = domain + hrefValue;
                }
                Object childValue = execute(fieldType, hrefValue, fieldObject);
                // 设置值
                field.set(object, childValue);
                continue;
            }

            // 获取Extract注解
            Extract extract = field.getAnnotation(Extract.class);
            if (extract != null) {
                // 不为空，需要抽取数据
                // 获取css选择器
                String fieldQuery = extract.cssQuery();
                // 抽取数据
                Elements fieldElement = element.select(fieldQuery);
                // 判断该字段的值是否要从属性获取
                String attr = extract.attr();
                if (StringUtils.isNotBlank(attr)) {
                    // 获取属性值，赋值给字段
                    String attrValue = fieldElement.attr(attr);
                    // 处理前缀
                    String prefix = extract.prefix();
                    attrValue = attrValue.replace(prefix, "");
                    // 赋值给字段
                    field.set(object, attrValue);
                } else {
                    // attr为空， 获取contentType， 决定如何取值
                    int contentType = extract.contentType();
                    if (contentType == 1) {
                        // 执行text
                        String text = fieldElement.text();
                        // 处理前缀
                        String prefix = extract.prefix();
                        text = text.replace(prefix, "");
                        // 赋值给字段
                        field.set(object, text);
                    } else if (contentType == 2) {
                        String html = fieldElement.html();
                        // 处理前缀
                        String prefix = extract.prefix();
                        html = html.replace(prefix, "");
                        // 赋值给字段
                        field.set(object, html);
                    } else {
                        // 存在了1和2以外的值，抛出异常
                        throw new RuntimeException("字段 " + field.getName() + " 抽取类型只能是1或2");
                    }
                }
            }
        }
    }

}
